from urllib.request import urlopen
from urllib.error import URLError
from urllib.error import HTTPError
import urllib.parse
import urllib.request
import time
import re

result_ = []

io_file = open('stars.txt', 'a', encoding='utf-8')

i=5
flag = False
try_times = 0
while (i < 20):
	# 尝试次数超过6次则跳过忽略本页i
	if (flag):
		try_times += 1
	if (try_times == 6):
		print(str(i) + "已被跳过!")
		try_times = 0
		i += 1

	print(str(i) + "start")
	#io_file.write(str(i) + " start \n")
	try:
		sum_req = urllib.request.Request("https://github.com/search?o=desc&p="
			+ str(i) + "&q=GITHUB.IO&s=stars&type=Repositories")
		sum_req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)') 
		# if has Chinese, apply decode()
		html = urlopen(sum_req).read().decode('utf-8')

		flag = False

		res = re.findall(re.compile(r'[a-zA-Z0-9-]+[.]github.io'), html)	

		# 去重
		res = list(set(res))
		# long time	
		for io in res:
			print("https://" + str(io))
			try:
				sin_req = urllib.request.Request("https://" + str(io))
				sin_req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)') 
				temp_html = urlopen(sin_req).read().decode('utf-8')

				reject = re.findall(re.compile(r'访问被拒绝'), temp_html)

				if len(reject) == 0:
					result_.append(str(io))
					print('ok!!!')
					#io_file.write(str(io) + "\n")

			except ConnectionResetError as e:
				print(e)
				continue
			except URLError as u:
				print(u)
				continue

	except ConnectionAbortedError as cae:
		print(str(cae) + ' :p wait 10 seconds...')
		flag = True
		time.sleep(10)
		continue
	except HTTPError as he:
		print(str(he) + ' :p wait 10 seconds...')
		flag = True
		time.sleep(10)
		continue
	except URLError as e:
		print(str(e) + ' :p wait 10 seconds...')
		flag = True
		time.sleep(10)
		continue

	print(str(i) + "end")
	#io_file.write(str(i) + " end \n")
	# time.sleep(31)
	result_ = list(set(result_))
	# print(result_)
	# io_file.write(str(result_) + " times " + str(i) + " end!!! \n")
	# io_file.close()
	# io_file = open('ok.txt', 'a', encoding='utf-8')
	i += 1


io_file.write(str(result_))


